import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.HTMLEditorKit.ParserCallback;
import javax.swing.text.html.parser.ParserDelegator;

/**
 * 
 *This program parse in the text and links HTML source code from provided link
 * and store distinct word that got written into a hashMap. The hashMap get
 * written in a file which get read by Lookup class.
 * 
 * @author Peter Tran (pxtran)
 * @author Andy Chhuseng Ieng (cieng) Copyright (C) 2009 Andy ChhuSeng Ieng
 *         <cieng@ucsd.edu> and Peter Tran <pxtran@ucsd.edu>
 * 
 */
public class Index {

	// mainHash map that get written in to the file
	private static HashMap<String, HashMap<String, Integer>> mainHash;

	// individual hash map that mapped to each distincted URL
	private static HashMap<String, HashMap<String, Integer>> individualHash;

	// store the text in between the A tags
	private static HashMap<String, String> anchorTextKeywordsHash;

	// track the visited link
	private static LinkedList<String> visitedListSearch;

	// track the first visited url for the word on the page itself
	private static LinkedList<String> firstVisitedList;

	// track the first visited url for the word on the page
	// itself and the anchor text
	private static LinkedList<String> secondVisitedList;

	// search flag for total page and the word on the page itself
	private static boolean firstSearch;

	// flag for storing the texts that were in between the tag
	private static boolean anchorFlag;

	/**
	 * constructor for index create visited listSearch and individual hash map
	 */
	public Index() {
		visitedListSearch = new LinkedList<String>();
		individualHash = new HashMap<String, HashMap<String, Integer>>();
	}

	/**
	 * Read in argument common line java Index firstpage searchtype maxpages
	 * indexfile\n Firstpage is the complete URL of the web page where the
	 * search should start, Searchtype is either breadth or depth, Maxpages is
	 * an integer specifying the maximum number of web pages to search Indexfile
	 * is the name of the file that will contain the index Read the URL and
	 * write the text content of the url source code in to a file.
	 * 
	 * @param args
	 *            java Index firstpage searchtype maxpages indexfile
	 * @throws MalformedURLException
	 */
	public static void main(String[] args) throws MalformedURLException {

		// create the index object access it function
		Index index = new Index();

		// check for valid argument length
		if (args.length < 4 || args.length > 4) {
			index.usage();
			System.exit(1);

		} else {

			// check for valid argument 2 for search type
			if ((!args[1].equals("breadth")) && (!args[1].equals("depth"))) {
				index.usage();
				System.exit(1);
			}

			// check for valid number
			for (int i = 0; i < args[2].length(); i++) {

				if (!Character.isDigit(args[2].charAt(i))) {

					index.usage();
					System.exit(1);

				}
			}

			// the complete URL to the web page where thesearch should start
			String firstpage = args[0];

			// searchtype breadth or depth
			String searchtype = args[1];

			// an integer specifying the maximun number of web pages to search
			int maxpages = Integer.parseInt(args[2]);

			// the name of the file that will contain the index
			String indexfile = args[3];

			// setting the first search flag the word on the page itself
			firstSearch = false;

			// setting the anchorFlag flag to add text in to visited url
			anchorFlag = false;

			// track the fist visited list
			firstVisitedList = new LinkedList<String>();

			// track the second visited list
			secondVisitedList = new LinkedList<String>();

			// track the mainHash to store in a file
			mainHash = new HashMap<String, HashMap<String, Integer>>();

			// track the text between the tag
			anchorTextKeywordsHash = new HashMap<String, String>();

			// try to catch the invalid url
			try {
				@SuppressWarnings("unused")
				URL myURL = new URL(firstpage);

//				if (firstpage.endsWith(".") || firstpage.endsWith("www")) {
//					index.usage();
//					System.exit(1);
//				}

			} catch (MalformedURLException e) {
				System.err.println(firstpage + " is not a valid absolute URL.\n");

				index.usage();
				System.exit(1);

			}

			System.out.println("Doing a " + searchtype
					+ "-first search of the Web, starting from");
			System.out.println(firstpage
					+ "\n-------------------------------------------------\n"
					+ "\tURL visited          (keywords on page itself)\n"
					+ "-------------------------------------------------");

			// looping through the list to get distinct keywords on itself
			while (firstVisitedList.size() < maxpages) {

				if (!firstVisitedList.contains(firstpage))

					// read in the url
					index.read(firstpage);

				if (!visitedListSearch.isEmpty()) {

					if (searchtype.equals("breadth")) {

						// breadth search url
						firstpage = visitedListSearch.removeFirst();

					} else if (searchtype.equals("depth")) {

						// depth search url
						firstpage = visitedListSearch.removeLast();

					}
				} else {
					// end the loop it nothing else to read
					break;
				}

			}

			firstSearch = true;

			System.out.println("Searched " + firstVisitedList.size()
					+ " web pages and found " + mainHash.size()
					+ " distinct keywords.");

			anchorFlag = true;

			mainHash = new HashMap<String, HashMap<String, Integer>>();

			System.out.println("In the keyword index:"
					+ "\n-------------------------------------------------"
					+ "\n\tURL visited          (total keywords)"
					+ "\n-------------------------------------------------");

			String nextpage = firstpage;

			// looping through the list to get distinct keywords on itself and
			// text between the tag
			while (secondVisitedList.size() < maxpages) {

				if (!secondVisitedList.contains(nextpage)) {

					if (anchorTextKeywordsHash.containsKey(nextpage)) {

						// get the string between the tag
						String anchorText = anchorTextKeywordsHash
								.get(nextpage);

						String[] items = anchorText.split("\\W");

						// parse in the text between the tag
						index.stringParser(items, nextpage);

					}

					// read in the url to get the distinct keywords
					index.read(nextpage);

				}

				if (!firstVisitedList.isEmpty()) {
					// get next page to get read
					nextpage = firstVisitedList.remove();
				} else {
					// end the loop it nothing else to read
					break;
				}

			}

			// clear out the list and hash map
			anchorTextKeywordsHash.clear();
			secondVisitedList.clear();

			System.out.println("Writing keyword index in file outfile... Done.");

			// write the mainHash map in to a file
			try {
				FileOutputStream file = new FileOutputStream(indexfile);
				index.write(indexfile, file);
				file.flush();
				file.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

		}
	}

	/**
	 * Read in the url and wash the text in the url source code into hash map.
	 * 
	 * @param nextpage
	 *            the page to get read
	 */
	public void read(String nextpage) {

		try {

			// URL string representative
			URL myURL = new URL(nextpage);

			// creating callback object
			ParserCallback callback = new HTMLParser(nextpage,
					visitedListSearch, anchorTextKeywordsHash, anchorFlag);

			// open the url connection
			Reader read = new BufferedReader(new InputStreamReader(myURL
					.openStream()));

			// new document parser
			HTMLEditorKit.Parser parser = new ParserDelegator();

			// get the results from the parse.
			parser.parse(read, callback, true);

			// track which list to use
			if (firstSearch == false) {
				firstVisitedList.add(nextpage);
			}
			if (firstSearch == true) {
				secondVisitedList.add(nextpage);
			}

			System.out.println(nextpage + " (" + individualHash.size() + ")");

			// clear out the old individual hash
			individualHash = new HashMap<String, HashMap<String, Integer>>();

			read.close();

		} catch (MalformedURLException e) {
			usage();
			System.exit(1);
		} catch (IOException e) {
		} catch (IllegalArgumentException x) {
		} catch (IllegalAccessError x) {
		}

	}

	/**
	 * Write the hashmap in the a file.
	 * 
	 * @param fileName
	 *            file name
	 * @param file
	 *            get which get written
	 * @throws IOException
	 */
	public void write(String fileName, FileOutputStream file)
			throws IOException {
		try {

			ObjectOutputStream output = new ObjectOutputStream(file);

			// get the main hash in to the file
			output.writeObject(mainHash);

			file.flush();

		} catch (Exception e) {// Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}

	/**
	 * USAGE MESSAGES
	 */
	public void usage() {
		System.err.println("USAGE:");
		System.err.println("\t\tjava Index firstpage searchtype"
				+ " maxpages indexfile\n");
		System.err.println("Firstpage is the complete URL of"
				+ "the web page where the search should start,"
				+ "\nSearchtype is either breadth or depth,"
				+ "\nMaxpages is an integer specifying the maximum"
				+ "number of web pages to search"
				+ "\nIndexfile is the name of the file"
				+ "that will contain the index");
	}

	/**
	 * Parse the String array to keyword to store in hash map along with the
	 * current URL.
	 * 
	 * @param items
	 *            string array
	 * @param workingURL
	 *            the current working URL
	 */
	public void stringParser(String[] items, String workingURL) {

		// track keyword
		String keyword = "";

		// added Hashmap that contain URL and distinct word count
		// into the MainHash
		HashMap<String, Integer> urlwordcountHash;

		// looping through the array to get the keyword
		for (String item : items) {

			// get the keyword
			keyword = Keyword.makeKeyword(item);

			// ignore empty string
			if (!keyword.equals("")) {

				// Check if word is in HashMap
				if (mainHash.containsKey(keyword)) {

					// get occurrences number for this word increment
					urlwordcountHash = mainHash.get(keyword);

					// check to see if the URl already exist in the map
					if (!urlwordcountHash.containsKey(workingURL)) {
						urlwordcountHash.put(workingURL, 1);
					} else {
						// increment the distinct keywords count
						urlwordcountHash.put(workingURL, urlwordcountHash
								.get(workingURL) + 1);
					}

					// get the individual
					individualHash.put(keyword, urlwordcountHash);

					// store keyword and it relative value
					mainHash.put(keyword, urlwordcountHash);

				} else {
					// track new value for the main hash map
					urlwordcountHash = new HashMap<String, Integer>();

					// store new URL
					urlwordcountHash.put(workingURL, 1);
					individualHash.put(keyword, urlwordcountHash);
					mainHash.put(keyword, urlwordcountHash);
				}
			}

		}

	}

	/**
	 * HTML parser proceeds from a callback for each and every piece of the
	 * HTML. Get the html in between the tag "A" to store in to a list which
	 * will get revisited later.
	 */
	class HTMLParser extends HTMLEditorKit.ParserCallback {

		// track visited URL
		HashMap<String, String> visitedText;

		// track the text between the tag
		HashMap<String, String> anchorTextKeywordsHash;

		// track the text between the tag
		String tagText = "";

		// track the end tag
		boolean startTag;

		// track the start tag
		boolean endTags;

		// track for valid URL
		boolean urlFlag;

		// track the anchor flag to store the text
		boolean anchorFlag;

		// the current URL
		String workingURL = "";

		// Visted list that will get visit later
		LinkedList<String> visitedListSearch;

		// tamp URL
		String urltag = "";
		String url = "";

		/**
		 * HTML Parser constructor initialized the anchor Text key word hash
		 * visited list and set all the flag.
		 * 
		 * @param myURL
		 *            the current URL
		 * @param visitedListSearch
		 *            visited list
		 * @param anchorTextKeywordsHash
		 *            the text between the tag
		 * @param anchorFlag
		 *            flag to signal the anchorTextKeywordsHash when to add the
		 *            text
		 * 
		 */
		public HTMLParser(String myURL, LinkedList<String> visitedListSearch,
				HashMap<String, String> anchorTextKeywordsHash,
				boolean anchorFlag) {
			this.anchorFlag = anchorFlag;
			startTag = false;
			endTags = false;
			urlFlag = false;
			this.workingURL = myURL;
			this.anchorTextKeywordsHash = anchorTextKeywordsHash;
			this.visitedListSearch = visitedListSearch;
		}

		/*
		 * (non-Javadoc)
		 * 
		 * @see
		 * javax.swing.text.html.HTMLEditorKit.ParserCallback#handleStartTag
		 * (javax.swing.text.html.HTML.Tag,
		 * javax.swing.text.MutableAttributeSet, int)
		 */
		public void handleStartTag(HTML.Tag tag, MutableAttributeSet attr,
				int pos) {

			// track the substring of the URL
			String subString = "";
			String currentUrl = "";

			if (tag == HTML.Tag.A) {

				// get the URL inside the A tag
				url = (String) attr.getAttribute(HTML.Attribute.HREF);

				
				if (url != null) {
					
					// check for valid URL
					for (int i = 0; i < url.length(); i++) {
						Character charToCheck = url.charAt(i);
						if (Character.isSpace((char) charToCheck)) {
							return;
						}
					}

					try {
						
						//check for absolute URL
						URL workingString = new URL(url);
						startTag = true;
						urlFlag = true;
						urltag = url;
						visitedListSearch.add(url);

					} catch (MalformedURLException e) {

						try {
							
							//check for relative URL
							if (workingURL.endsWith("/")) {
								currentUrl = workingURL + url;
							} else {
								
								//getting the sub string of the absolute URL
								if (!url.startsWith("/")) {
									subString = workingURL.substring(0,
											workingURL.lastIndexOf("/") + 1);
								}
								currentUrl = subString + url;
							}

							//check the relative URL
							URL workingString = new URL(currentUrl);

							startTag = true;
							urlFlag = true;

							urltag = currentUrl;
							
							//Storing the ralative url to list
							visitedListSearch.add(currentUrl);

						} catch (MalformedURLException msg) {
							// System.err.println("HTTP 403");
						}
					}

				}
			}

		}

		/*
		 * (non-Javadoc)
		 * 
		 * @see
		 * javax.swing.text.html.HTMLEditorKit.ParserCallback#handleEndTag(javax
		 * .swing.text.html.HTML.Tag, int)
		 */
		public void handleEndTag(HTML.Tag tag, int pos) {
			
			//check for end tag "A"
			if (tag == HTML.Tag.A) {
				endTags = true;
				startTag = false;
			}

		}

		/*
		 * (non-Javadoc)
		 * 
		 * @see
		 * javax.swing.text.html.HTMLEditorKit.ParserCallback#handleText(char[],
		 * int)
		 */
		public void handleText(char[] data, int pos) {

			//get the bundle string
			String text = new String(data);
			
			//check to get the text between the start tag and end tag
			if (startTag == true && endTags == false) {
				
				//connect the text between the start tag and the end tag
				//together if any.
				if (urlFlag == true) {
					tagText += " " + text;
					if (anchorFlag == false) {
						//adding the text and the corresponding url
						anchorTextKeywordsHash.put(urltag, tagText);
					}
				}
				
			} else {
				
				tagText = "";
				urlFlag = false;
				startTag = false;
				endTags = false;
			}
			
			//splitting the bundle string when it find not a character 
			//and number object in the string
			String[] items = text.split("\\W");
			
			//call string parser to parse in the string bundle
			stringParser(items, workingURL);
		}

	}
}
